import random

import redis
import requests

def should_request(url):
    redis.client = redis.Redis(host='localhost', port=6379, db=0)
    urls = redis.client.lrange('tasks', 0, -1)
    return url.encode('utf-8') in urls


def save_redis(url):
    redis.client = redis.Redis(host='localhost', port=6379, db=0)
    redis.client.lpush('tasks',url)
    redis.client.close()
urls = [f'https://httpbin.org/get?age={random.randint(1, 20)} ' for _ in range(10)]
urls = set(urls)
for url in urls:
    if not should_request(url):
        response = requests.get(url)
        print(response.status_code)
        save_redis(url)
    else:
        print("不需要爬取")